/**
 * @(#) QiDianNovelAcquisition.java
 *
 * @author lihg
 * @version V1.0 2013-10-4
 *
 * Copyright (C) 2010,2013 , Lihg
 */

/**
 * 
 */
package cn.com.dshome.acquisition.service.impl;

import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;

import cn.com.dshome.acquisition.dao.novel.NovelArticlesDao;
import cn.com.dshome.acquisition.dao.novel.NovelAuthorsDao;
import cn.com.dshome.acquisition.dao.novel.NovelCategoriesDao;
import cn.com.dshome.acquisition.dao.novel.NovelChaptersDao;
import cn.com.dshome.acquisition.entity.novel.NovelArticles;
import cn.com.dshome.acquisition.entity.novel.NovelAuthors;
import cn.com.dshome.acquisition.entity.novel.NovelCategories;
import cn.com.dshome.acquisition.entity.novel.NovelChapters;
import cn.com.dshome.acquisition.exception.AcquisitionException;
import cn.com.dshome.acquisition.service.Acquisition;
import cn.com.dshome.acquisition.utils.HttpClientUtils;
import cn.com.dshome.acquisition.utils.StringUtils;
import cn.com.dshome.acquisition.vo.BaseInformation;
import cn.com.dshome.acquisition.vo.QiDianNovelInformation;

/**
 * 
 * Purpose:
 * 
 * @author lihg
 * @see
 * @since
 */
@Service("novelAcquisition")
public class QiDianNovelAcquisition implements Acquisition {

	private static final long serialVersionUID = 3890664452145997343L;

	private String url = "http://www.qidian.com/Book/BookStore.aspx?OrderId=1" ;//"http://read.qidian.com/BookReader/2952453.aspx" ;

	//内容正则表达式
	private Pattern contentPattern = Pattern.compile(".*?\\('(.*?)'\\).*?") ;

	//记录日志
	private Log log = LogFactory.getLog(QiDianNovelAcquisition.class) ;

	@Autowired
	@Qualifier("httpClientUtils")
	private HttpClientUtils hcu ;

	@Autowired
	private NovelCategoriesDao novelCategoriesDao ;

	@Autowired
	private NovelArticlesDao novelArticlesDao ;

	@Autowired
	private NovelAuthorsDao novelAuthorsDao ;

	@Autowired
	private NovelChaptersDao novelChaptersDao ;

	@Autowired
	private TaskExecutor taskExecutor ;

	/* (non-Javadoc)
	 * @see cn.com.dshome.acquisition.service.Acquisition#listInformation(cn.com.dshome.acquisition.vo.BaseInformation)
	 */
	@Override
	public <T extends BaseInformation> List<T> listInformation(Class<T> clazz)
	throws AcquisitionException {
		try {
			//获取地址列表
			List<String> urlList = new ArrayList<String>() ;
			fetchAllUrl(urlList) ;
			System.out.println("共：【" + urlList.size() + "】篇小说");
			for(final String bookUrl : urlList){
				//异步获取图书相关信息
				taskExecutor.execute(new Runnable() {
					@Override
					public void run() {
						try {
							fetchMessage(bookUrl) ;
						} catch (AcquisitionException e) {
							log.error("获取失败!" , e) ;
						}
					}
				}) ;
			}
		} catch (Exception e) {
			log.error(e) ;
		}
		return null;
	}

	/**
	 * 获取所有的URL地址
	 * @param urlList
	 * @throws AcquisitionException
	 */
	private void fetchAllUrl( List<String> urlList) throws AcquisitionException{
		String responseMessage = convertByteToString(hcu.getResponseMessage(url)) ;
		Pattern pattern = Pattern.compile("initPageCount='(.*?)'") ;
		Matcher matcher = pattern.matcher(responseMessage) ;
		Integer totalPage = 1 ;
		while(matcher.find()){
			try {
				totalPage = Integer.valueOf(matcher.group(1));
			} catch (Exception e) {
				log.error("获取总页数失败！",e) ;
			}
		}
		Document document = Jsoup.parse(responseMessage) ;
		fetchBookUrl(document,urlList) ;
		//获取各页图书的URL地址
		for(int pageIndex = 2 ; pageIndex <= totalPage ; pageIndex ++ ){
			String pageBookUrl = "http://all.qidian.com/book/bookstore.aspx?ChannelId=-1&SubCategoryId=-1&Tag=all&Size=-1&Action=-1&OrderId=1&P=all&update=-1&Vip=-1&Boutique=-1&SignStatus=-1&PageIndex=" + pageIndex ;
			Document pageDocument = Jsoup.parse(convertByteToString(hcu.getResponseMessage(pageBookUrl)));
			fetchBookUrl(pageDocument,urlList) ;
		}
	}

	/**
	 * 获取图书地址
	 * @param document
	 * @param bookUrl
	 */
	private void fetchBookUrl(Document document,List<String> bookUrl){
		Elements elements = document.select("span.swbt>a") ;
		for(Element element : elements){
			String url = element.attr("href") ;
			url = "http://read.qidian.com/BookReader/" + url.substring(url.indexOf("/",5) + 1);
			bookUrl.add(url) ;
			log.info("图书：【" + element.html() + "】URL：【" + url + "】") ;
		}
	}



	/**
	 * 获取需要采集的信息（采集入库）
	 * @return QiDianNovelInformation
	 * @throws AcquisitionException
	 */
	private QiDianNovelInformation fetchMessage(String url) throws AcquisitionException{
		//第一次返回信息
		String responseMessage = convertByteToString(hcu.getResponseMessage(url) );
		//解析返回的HTML信息
		Document document = Jsoup.parse(responseMessage) ;
		//文章内容开始（阅读界面） id=bookdirectory
		Element bookDirectory = document.getElementById("bookdirectory") ;
		//文章标题 class=booktitle
		//文章基本信息
		Element bookTitle = bookDirectory.getElementsByClass("booktitle").get(0).siblingElements().get(0) ;
		//文章主信息
		QiDianNovelInformation qdniMain = new QiDianNovelInformation() ;
		//设置文章描述信息
		qdniMain.setContent(bookTitle.getElementsByAttributeValue("itemprop","description").get(0).html()) ;
		//设置文章分类
		qdniMain.setCategory(bookTitle.getElementsByAttributeValue("itemprop","genre").get(0).html()) ;
		//设置字数
		qdniMain.setCount(Integer.valueOf(bookTitle.getElementsByAttributeValue("itemprop","wordCount").get(0).text())) ;
		//设置作者
		qdniMain.setAuthor(bookTitle.getElementsByAttributeValue("itemprop","author").get(0).getElementsByAttributeValue("itemprop","name").get(0).text()) ;
		//设置发布人
		qdniMain.setPublisher(qdniMain.getAuthor()) ;
		//设置主题
		qdniMain.setArticle(bookTitle.getElementsByAttributeValue("itemprop","name").get(0).text()) ;
		//图书种类
		NovelCategories category = new NovelCategories() ;
		category.setName(qdniMain.getCategory()) ;
		//增加图书种类
		category.setId(novelCategoriesDao.saveOrDoNothing(category).toString()) ;
		//作者信息
		NovelAuthors author = new NovelAuthors() ;
		author.setName(qdniMain.getAuthor()) ;
		//增加作者信息
		author.setId(novelAuthorsDao.saveOrDoNothing(author).toString()) ;
		//文章信息
		NovelArticles article = new NovelArticles() ;
		article.setAuthor(author) ;
		article.setCategory(category) ;
		article.setName(qdniMain.getArticle()) ;
		article.setWordCount(qdniMain.getCount()) ;
		article.setDescription(qdniMain.getContent()) ;
		//增加文章信息
		article.setId(novelArticlesDao.saveOrDoNothing(article).toString()) ;
		log.info(qdniMain);
		//文章详细内容区
		Element content = bookDirectory.getElementById("content") ;
		//获取所有连接
		Elements list = content.select("div.box_cont>div.list>ul>li>a") ;
		//普通章节信息直接从起点中文网上获取
		final List<QiDianNovelInformation> normalInformations = new ArrayList<QiDianNovelInformation>() ;
		//VIP章节信息从来书网上获取
		final List<QiDianNovelInformation> vipInformations = new ArrayList<QiDianNovelInformation>() ;
		for(final Element anchor : list){
			//异步遍历章节信息
			taskExecutor.execute(new Runnable() {
				@Override
				public void run() {
					QiDianNovelInformation information = new QiDianNovelInformation() ;
					String href = anchor.attr("href") ;
					if(!href.startsWith("http:")){//非VIP章节
						href = "http://read.qidian.com" + href ;
						//设置Url地址
						information.setUrl(href) ;
						//设置分类标题
						information.setArticle(anchor.getElementsByAttributeValue("itemprop","headline").html()) ;
						//非VIP章节
						information.setIsVIPChapter(false) ;
						normalInformations.add(information) ;
					}else{//VIP章节
						//fetFromLaiShu.com
						//设置文章标题
						information.setArticle(anchor.html()) ;
						String title = anchor.attr("title") ;
						int index = title.indexOf(" ",20) ;
						//设置发布时间
						information.setPublishTime(title.substring(5,index)) ;
						//设置简介
						information.setContent(title.substring(index+1)) ;
						//VIP章节
						information.setIsVIPChapter(true) ;
						vipInformations.add(information) ;
					}
				}
			}) ;
		}
		//获取VIP章节详细信息
		fetchVipInformationFromLaiShu(article.getName(),vipInformations) ;
		normalInformations.addAll(vipInformations) ;
		//获取章节详细信息
		fetchChapters(article,normalInformations) ;
		qdniMain.setSubInformations(normalInformations) ;
		return qdniMain ;
	}

	/**
	 * 获取VIP章节URL地址
	 * @param novelName
	 * @param vipInformations
	 * @throws AcquisitionException
	 */
	private void fetchVipInformationFromLaiShu(String novelName,List<QiDianNovelInformation> vipInformations) throws AcquisitionException{
		try {
			String encode = URLEncoder.encode(novelName,"GBK") ;
			//直接来书站内搜索
			String url = "http://www.laishu.com/modules/article/search.php?searchkey=" + encode + "&searchtype=articlename&searchbuttom=%CB%D1%CA%E9";
			//获取书详细信息
			Document contentDocument = Jsoup.parse(convertByteToString(hcu.getResponseMessage(url))) ;
			//获取章节节点
			final Elements nodesList = contentDocument.select("td.listbg>a") ;
			//章节节点匹配
			//			final Pattern chapterPattern = Pattern.compile("(.*)\\s.*?") ;
			for(final QiDianNovelInformation vipInformation : vipInformations){
				//异步设置VIP章节URL地址
				taskExecutor.execute(new Runnable() {
					@Override
					public void run() {
						String article = vipInformation.getArticle() ;
						//						Matcher chapterMatcher = chapterPattern.matcher(article) ;
						//						if(chapterMatcher.find()){
						//							article = chapterMatcher.group(1) ;
						//						}
						for(Element anchor : nodesList){
							if(article.indexOf(anchor.html()) != -1){
								String subUrl = "http://www.laishu.com" + anchor.attr("href") ;
								vipInformation.setUrl(subUrl) ;
								break ;
							}
						}
					}
				}) ;
			}
		} catch (Exception e) {
			log.error(e) ;
			throw new AcquisitionException(e) ;
		}
	}
	/**
	 * 获取章节信息
	 * @param novelArticles
	 * @param informationList
	 */
	private void fetchChapters(final NovelArticles novelArticles,List<QiDianNovelInformation> informationList){
		for(final QiDianNovelInformation information : informationList){
			//异步获取章节信息
			taskExecutor.execute(new Runnable() {
				@Override
				public void run() {
					//章节信息
					NovelChapters chapter = new NovelChapters() ;
					chapter.setArticle(novelArticles) ;
					chapter.setName(information.getArticle()) ;
					if(! novelChaptersDao.isHaveThisChapter(chapter)){
						log.info("文章：【"  + novelArticles.getName() + "】 未包含章节【" + chapter.getName() + "】，获取中……") ;
						try {
							fetchChapters(information) ;
							chapter.setWordCount(information.getCount()) ;
							chapter.setUpdateTime(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(information.getPublishTime())) ;
							chapter.setContent(information.getContent().getBytes()) ;
							novelChaptersDao.save(chapter) ;
							//							novelArticles.setLastUpdateTime(chapter.getUpdateTime()) ;
							//							novelArticlesDao.update(novelArticles) ;
						} catch (Exception e) {
							log.error(e) ;
						}
					}else{
						log.info("文章：【"  + novelArticles.getName() + "】 已包含章节【" + chapter.getName() + "】，不再获取") ;
					}
				}
			}) ;
		}
	}


	/**
	 * 获取章节信息
	 * @param information
	 * @throws AcquisitionException
	 */
	private void fetchChapters(QiDianNovelInformation information) throws AcquisitionException{
		if( ! information.getIsVIPChapter()){
			fetchNormalChapters(information) ;
		}else{
			fetchVIPChapters(information) ;
		}
	}

	/**
	 * 获取普通章节信息
	 * @param information
	 * @throws AcquisitionException
	 */
	private void fetchNormalChapters(QiDianNovelInformation information) throws AcquisitionException{
		//解析返回的HTML信息
		Document document = Jsoup.parse(convertByteToString(hcu.getResponseMessage(information.getUrl())));
		//获取主元素
		Element mainContent = document.getElementById("maincontent") ;
		//获取基本信息
		//标题
		//		Element article = mainContent.getElementById("lbChapterName") ;
		//
		//		information.setArticle(article.html()) ;

		//更新时间
		Element updateTime = mainContent.getElementById("lblLastUpdateTime") ;
		information.setPublishTime(updateTime.html()) ;
		//字数
		Element wordCount = mainContent.getElementById("lbnWordCount") ;
		information.setCount(Integer.valueOf(wordCount.text())) ;
		//内容
		Element content = mainContent.getElementById("content") ;
		//获取具体信息
		String details =  convertByteToString(hcu.getResponseMessage(content.child(0) .attr("src")));
		Matcher matcher = contentPattern.matcher(details) ;
		while(matcher.find()){
			information.setContent(matcher.group(1)) ;
		}
	}
	/**
	 * 获取VIP章节详细信息
	 * @param information
	 * @throws AcquisitionException
	 */
	private void fetchVIPChapters(QiDianNovelInformation information) throws AcquisitionException {
		//获取响应信息
		String responseMessage = convertByteToString(hcu.getResponseMessage(information.getUrl()));
		//获取详细信息
		Document contentDocument = Jsoup.parse(responseMessage) ;
		//获取章节内容
		Element content = contentDocument.getElementById("txt_td") ;
		if(null != content){
			information.setContent(content.html().replaceAll("<a.*?>好看的小说</a>","")) ;
		}
		Integer count = information.getCount() ;
		String contentCount = information.getContent() ;
		if(null == count){
			count = contentCount.replaceAll("\\&.*?;|<.*?>|好看的小说|\\s*","").length() ;
		}
		//设置字数
		information.setCount(count) ;
	}

	/**
	 * Byte转String
	 * @param content
	 * @return
	 */
	private String convertByteToString(byte[] content){
		try {
			String charSet = hcu.getChartSet() ;
			if (StringUtils.hasLength(charSet)) {
				return StringUtils.byteArray2String(content, charSet);
			} else {
				return StringUtils.byteArray2String(content);
			}
		} catch (Exception e) {
			log.error("convertByteToString failed " ,e) ;
			return null ;
		}
	}
}


/**
 * Revision history
 * -------------------------------------------------------------------------
 * 
 * Date Author Note
 * -------------------------------------------------------------------------
 * 2013-10-4 上午10:04:06 lihg 创建版本
 */